<!DOCTYPE html>
<!--
     SPDX-License-Identifier: CC-BY-SA-4.0
     SPDX-FileCopyrightText: 2020 seL4 Project a Series of LF Projects, LLC.
-->
<!-- Page last generated 2025-02-20 03:16:15 +0000 -->
<html lang="en">
  <head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <title>ZeroMQ examples | seL4 docs</title>

    <!-- Our stylesheet and theme stylesheet.  Contains bootstrap. -->
    <link rel="stylesheet" href="/assets/css/style.css" type="text/css">
    <!-- Font awesome -->
    <link href="https://use.fontawesome.com/releases/v5.0.8/css/all.css" rel="stylesheet">
    <link href="https://fonts.googleapis.com/css2?family=Roboto&display=swap" rel="stylesheet">
    <!-- Pygments syntax highlighting  -->
    <link rel="stylesheet" href="/assets/css/highlighting/trac.css" type="text/css">
    <link rel="icon" type="image/x-icon" href="/assets/favicon.ico"><script defer data-domain="docs.sel4.systems"
	    src="https://analytics.sel4.systems/js/script.js"></script></head>

  <body class="container-fluid">

    



<header>
  <ul class="row menu">
    <li class="col-xs-12 col-md-2" >
            <a href="https://sel4.systems" class="skip-icon">
              <img class="img-responsive" src="/assets/logo-text-white.svg" alt="seL4 logo" />
            </a>
    </li>
    <li class="col-xs-12 col-md-10 menu">
      <nav aria-label="Banner links">
        <h2><a href="/Resources" />Resources</h2>
        <h2><a href="/processes" />Contributing</a></h2>
        <h2><a href="/projects" />Projects</h2>
        <h2><a href="/Tutorials" />Tutorials</h2>
        <iframe title="DuckDuckGo search bar" src="https://duckduckgo.com/search.html?site=docs.sel4.systems&prefill=Search%20sel4.systems" style="overflow:hidden;margin-bottom:10px; padding:0;height:40px;float:right;border-width: 0px"></iframe>
      </nav>
    </li>
  </ul>
  <div class="clear"></div>
  
<div class="breadcrumbs bootstrap hidden-sm-down">
  <nav class="sel-breadcrumb" aria-label="Breadcrumb" >
    <ol class=" list-unstyled" vocab="http://schema.org/" typeof="BreadcrumbList">
      
      
        

        

        <li class="breadcrumb-item" property="itemListElement" typeof="ListItem">
            <a property="item" typeof="WebPage" href="/">
              <span property="name"><b>seL4 Docs</b></span>
            </a>
            <meta property="position" content="1" />
        </li>
      
        

        

        <li class="breadcrumb-item" property="itemListElement" typeof="ListItem">
            <a property="item" typeof="WebPage" href="/projects/">
              <span property="name"><b>Projects</b></span>
            </a>
            <meta property="position" content="2" />
        </li>
      
        

        

        <li class="breadcrumb-item" property="itemListElement" typeof="ListItem">
            <a property="item" typeof="WebPage" href="/projects/camkes-vm/">
              <span property="name"><b>CAmkES VMM</b></span>
            </a>
            <meta property="position" content="3" />
        </li>
      
        

        
          <li class="breadcrumb-item" property="itemListElement" typeof="ListItem">
            <span property="name">ZeroMQ examples</span>
            <meta property="position" content="4" /></li>
          
    </ol>
  </nav>
  <nav class="sel-version" aria-label="Current Versions">
    <ol class="list-unstyled">
      <li class="list-unstyled text-right" style="margin-left:auto; padding:0rem 0rem;">
        Current versions:</li>
      <li class="list-unstyled text-right">
      <a href="/releases/sel4/13.0.0"><b>seL4-13.0.0</b></a></li>
      <li class="list-unstyled text-right">
      <a href="/releases/microkit/1.4.1"><b>microkit-1.4.1</b></a></li>
      <li class="list-unstyled text-right">
      <a href="/releases/camkes/camkes-3.11.0"><b>camkes-3.11.0</b></a></li>
      <li class="list-unstyled text-right">
      <a href="/releases/capdl/0.3.0"><b>capDL-0.3.0</b></a></li>
      </ol>
  </nav>
  <div class='clear'></div>
</div>


</header>

    <main>
      <div class="row">
  <div class="hidden-xs col-sm-4 col-md-3 col-lg-2">
    


<div class="sidebar">















    <ul class="nav nav-sidebar">
  
    
    

    
        <li class="">
            <a class="" href="/projects/sel4/">seL4</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/l4v/">L4.verified</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/camkes/">CAmkES</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/capdl/">CapDL</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/elfloader/">Elfloader</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/buildsystem/">seL4 Buildsystem</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/user_libs/">user_libs</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/sel4runtime/">The seL4 run-time</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/sel4webserver/">seL4webserver</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/sel4-tutorials/">seL4 tutorials</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/sel4bench/">sel4bench</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/sel4test/">seL4Test</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/virtualization/">Virtualization</a>
        </li>
  
    
    

    
        <li class="active">
            <a class="" href="/projects/camkes-vm/">camkes-vm</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/dockerfiles/">Dockerfiles</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/docsite/">seL4 Documentation website</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/sel4_tools/">seL4_tools</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/microkit/">Microkit</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/hardware_hacks/">Hardware Hacks</a>
        </li>
  
    
    

    
        <li class="">
            <a class="" href="/projects/rust/">Rust</a>
        </li>
  
    </ul>


</div>

  </div>
  <div class="content col-sm-8 col-md-6 col-lg-7 main">
    
<!--
  Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)

  SPDX-License-Identifier: CC-BY-SA-4.0
-->

<h1 id="zeromq-cross-vm-communication-example">ZeroMQ cross-vm communication example</h1>

<p>The <code class="language-plaintext highlighter-rouge">zmq_samples</code> application demonstrates messaging between VMs using the ZeroMQ messaging library. Each VM contains a VirtNet
ethernet device <code class="language-plaintext highlighter-rouge">eth0</code> which is connected to the <code class="language-plaintext highlighter-rouge">eth0</code> devices in the other VMs. Ethernet packets sent from a VM over this
interface will be transmitted to another VM if the correct macaddress is used. Broadcasting is supported which enables ARP to facilitate Macaddress discovery from an IP.</p>

<p>There are three ZeroMQ application scenarios.  They can be found in zmq_samples/src/user:</p>
<ul>
  <li>Client/server: A client program that connects to a server program over a network connection and performs a series of requests</li>
  <li>Publish/subscribe: A publish program that publishes information to a network address, and a subscribe program that connects to this address and receives published information</li>
  <li>pipeline: 3 programs, a source, worker and sink, that generate, process and acumulate computational tasks.</li>
</ul>

<p>The CMakeLists.txt file in zmq_samples builds libzmq (likely found in tools/libzmq) for a linux 32bit target environment, builds the above apps, and installs them into the buildroot rootfs using CMake functions declared found in camkes-vm-linux.
An custom init script is installed for each of the VMs that run each of the scenarios on a single VM, and then accross all 3 VMs, before providing the login prompt for each of them.  Currently if all of the scenarios are run correctly, <code class="language-plaintext highlighter-rouge">Finished running all scenarios</code> will be printed, otherwise <code class="language-plaintext highlighter-rouge">Exited with error</code> on an error. If one of the VMs appear to have hung, this could indicate an incorrect network address.</p>

<p>Each network interface is configured as follows:</p>

<table>
  <thead>
    <tr>
      <th>vm</th>
      <th>IP</th>
      <th>MAC</th>
      <th>IF</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>vm0</td>
      <td>192.168.1.1</td>
      <td>02:00:00:00:AA:01</td>
      <td>eth0</td>
    </tr>
    <tr>
      <td>vm1</td>
      <td>192.168.1.2</td>
      <td>02:00:00:00:AA:02</td>
      <td>eth0</td>
    </tr>
    <tr>
      <td>vm2</td>
      <td>192.168.1.3</td>
      <td>02:00:00:00:AA:03</td>
      <td>eth0</td>
    </tr>
  </tbody>
</table>

<h2 id="checkout-and-build">Checkout and build</h2>

<p>See <a href="/GettingStarted.html#setting-up-your-machine">Setting up your machine</a> for instructions on how to set up your host machine.</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>repo init -u https://github.com/sel4/camkes-vm-examples-manifest.git -b tipc-stage

repo sync

mkdir build-zmq

cd build-zmq

../init-build.sh -DCAMKES_VM_APP=zmq_samples

ninja
</code></pre></div></div>
<p>Then it should be able to be run on a haswell-like machine.</p>

<p>The working output is below.</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>Running 3 scenarios on one vm
emul_raw_tx@virtio_net_vswitch.c:146Running client server scenario
 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
message size: 1 [B]
roundtrip count: 10000
average latency: 20.153 [us]
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:ff:0:aa:3. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
Running pub/sub scenario
6
Collecting updates from weather server…
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:16. Dropping frame.
temp: 93901 -25 36 1
temp: 95278 -19 30 2
temp: 52299 34 29 3
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
temp: 94794 33 10 4
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
temp: 65265 16 46 5
Average temperature for zipcode was 7F
Running pipeline scenario
Sending tasks to workers…
Total expected cost: 5021 msec
10.69.81.9.49.:...86..5..13..11..4..77..91.:97..38..74..10..94..89..64..53..15..32.:50..85..34..42..69..emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
emul_raw_tx@virtio_net_vswitch.c:146 Unreachable dest macaddr 33:33:0:0:0:2. Dropping frame.
86..98..79..8..72.:47..89..80..96..74..85..8..85..88..84.:75..84..22..48..94..15..36..57..68..51.:88..17..36..22..58..5..7..56..83..15.:28..30..3..8..25..77..92..32..61..80.:16..35..64..37..83..57..52..19..14..19.:69..2..36..5..23..93..9..30..49..92.:44..76..21..47..83..46..23..74...Total elapsed time: 5364 msec
Running 3 scenarios accross 3 vms
Running client server scenario
[   15.529525] random: crng init done
[   16.555934] random: crng init done
message size: 1 [B]
roundtrip count: 10000
average latency: 87.514 [us]
Collecting updates from weather server…
Running pub/sub scenario
6
temp: 74943 -21 27 1
temp: 37849 -33 59 2
temp: 30536 67 51 3
temp: 98210 -11 30 4
temp: 56436 -39 20 5
Average temperature for zipcode was -7F

Welcome to Buildroot
buildroot login: Running pipeline scenario
Sending tasks to workers…
Total expected cost: 5494 msec

Welcome to Buildroot
buildroot login: 98.42.49.79.40.29.40.32.90.97.63.18.78.25.29.56.84.32.59.46.84.38.4.35.22.57.70.55.95.51.30.48.94.58.72.53.99.83.61.4.90.32.46.58.22.10.33.46.49.75.100.95.:.........:.........:.........:.........:.........39.8.82.88.50.77.66.81.21.36.47.93.50.10.22.52.18.91.63.32.41.79.40.94.60.73.97.18.53.36.78.53.:.........:.........:.........:.........:.........Total elapsed time: 3282 msec
Finished running all scenarios

Welcome to Buildroot
buildroot login:
</code></pre></div></div>

<h4 id="changing-connection-topology">Changing connection topology</h4>

<p>The topology definition can be found in zmq_samples.camkes.</p>
<ul>
  <li>To change the macaddress of a vm, <code class="language-plaintext highlighter-rouge">VM[ID]_MACADDRESS</code> needs to be updated</li>
  <li>To change what vms a vm connects to: the vms need to be removed for each other’s <code class="language-plaintext highlighter-rouge">topology_def</code></li>
  <li>To create partitioned topologies, a separate topology_def needs to be created and separate calls to each of the macros in the file will need to be added</li>
  <li>For VMs that are in the same topology, it may be possible for vms to read ethernet frames meant for other vms in the same topology.</li>
  <li>If a vm is used in more than one topology it will still only have one network interface, but vms from one topology won’t be able to read frames sent over a separate topology.</li>
  <li>It is not currently possible for two vms to be connected to each other in more than one topology.  It is intended that this
will be changed if a topology becomes tied to a separate network interface in the future.</li>
</ul>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>// VM0 is connected to vm1 and vm2
#define VM0_topology_def(f) f(0,1,2)

// VM1 is connected to vm0 and vm2
#define VM1_topology_def(f) f(1,0,2)

// VM2 is connected to vm0 and vm1
#define VM2_topology_def(f) f(2,0,1)

// topology_def is passed into all of the macros below
#define topology_def(f) \
    VM0_topology_def(f) \
    VM1_topology_def(f) \
    VM2_topology_def(f)

// MAC Addresses that each eth0 will have.
#define VM0_MACADDRESS "02:00:00:00:AA:01"
#define VM1_MACADDRESS "02:00:00:00:AA:02"
#define VM2_MACADDRESS "02:00:00:00:AA:03"

component Init0 {
    VM_INIT_DEF()

    // This declares the send and receive interfaces for each connection from vm0
    // This needs to be called in each Init* component that is referred to in the topology_def
    VM_CONNECTION_COMPONENT_DEF(0,topology_def)
}

// A dummy component to serve as the to side of the connection that we create.
component virtqueueinit {
    provides VirtQueue q;
}

// ... Other component declarations

assembly {
    composition {
        // ... The rest of the composition

        // The instance of the dummy component
        component virtqueueinit init;

        // The connection that connects all of the interfaces together.
        VM_CONNECTION_CONNECT_VMS(init.q, topology_def)

    }

    configuration {
        // Specific configuration
        VM_CONNECTION_CONFIG(init.q, topology_def)

        vm0.init_cons = [
            /* Initialise VirtNet eth0 device in vm0 */
            VM_CONNECTION_INIT_HANDLER,
        ];
        // ... The rest of the configuration

    }
}

</code></pre></div></div>

<h4 id="underlying-implementation">Underlying implementation</h4>
<p>The underlying mechanisms/interfaces used to implement cross vm communication are:</p>
<ul>
  <li>projects/vm/components/VM/configurations/connections.h: Contains the macro API used in zmq_samples.camkes</li>
  <li>projects/projects_libs/libvirtqueue: Virtqueue implementation that takes buffers from a restricted address space.</li>
  <li>projects/projects_libs/libvswitch: A library that resolves macaddresses to virtqueues with broadcast support.</li>
  <li>projects/vm/components/Init/virtio_net_vswitch.c: VirtioNet vmm device.  Uses libvirtqueue, libvswitch and libsel4camkes
to implement cross component communication</li>
  <li>projects/vm/templates/seL3VirtQueues-from.template.c: Camkes template for a virtqueue connection end.</li>
  <li>tools/camkes/libsel4camkes: Provides camkes bindings for libvirtqueue virtqueues backed by Camkes connector objects.</li>
</ul>

<h2 id="cakeml-filter-variant">CakeML Filter Variant</h2>

<p>A variant of this app that uses a CakeML component in place of the 3rd VM can
be built by providing an additional option to the build script:</p>

<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>$ ../init-build.sh -DCAMKES_VM_APP=zmq_samples -DCAKEML_FILTER=TRUE -DCAKEMLDIR=/path/to/cakeml
</code></pre></div></div>

<p>The CakeML component sits between the two VMs and proxies their traffic to each
other. Presently it doesn’t do any filtering.</p>

<p>The following versions of CakeML and HOL are known to work:</p>

<ul>
  <li>CakeML: 66a35311787bb43f72e8e758209a4745f288cdfe</li>
  <li>CakeML Binary Compiler: 1bb8663a0aeb377eb22aeb600de4bc536e647744</li>
  <li>HOL: 7323105f50960bdec1b33c513576e5d1d313b62f</li>
</ul>

<p><em>File included from <a href="https://github.com/sel4/camkes-vm-examples/blob/master/apps/x86/zmq_samples/README.md">github repo</a> <a href="https://github.com/sel4/camkes-vm-examples/edit/master/apps/x86/zmq_samples/README.md">edit</a></em></p>


  </div>







  
    
<div class="sidebar-toc hidden-xs hidden-sm col-md-3 col-lg-3">
    <ul id="toc" class="section-nav">
<li class="toc-entry toc-h2"><a href="#checkout-and-build">Checkout and build</a>
<ul>
<li class="toc-entry toc-h4"><a href="#changing-connection-topology">Changing connection topology</a></li>
<li class="toc-entry toc-h4"><a href="#underlying-implementation">Underlying implementation</a></li>
</ul>
</li>
<li class="toc-entry toc-h2"><a href="#cakeml-filter-variant">CakeML Filter Variant</a></li>
</ul>
</div>

  
  
<div class="sidebar-toc hidden-xs hidden-sm col-md-3 col-lg-3">
  
    <ul class="section-nav">
    	<h2> camkes-vm </h2> 
        <li>
          
          <a style="" class="" href="/projects/camkes-vm/">
            Documentation homepage
          </a>
        </li>

















    
        <h3>Repositories</h3>
    
        <li>
          <a class="" href="https://github.com/seL4/camkes-vm">
            camkes-vm
          </a>
        </li>

    
        <li>
          <a class="" href="https://github.com/seL4/camkes-vm-examples">
            camkes-vm-examples
          </a>
        </li>

    
        <li>
          <a class="" href="https://github.com/seL4/camkes-vm-examples-manifest">
            camkes-vm-examples-manifest
          </a>
        </li>









  







    
        <h3>Recent Updates</h3>
    
        <li>
          <a style="" href="/updates/camkes-vm/camkes-3.8.x-arm-old.html">
            camkes-3.8.x-compatible-arm-old
          </a>
        </li>

    
        <li>
          <a style="" href="/updates/camkes-vm/camkes-3.8.x.html">
            camkes-3.8.x-compatible
          </a>
        </li>

    </ul>

</div>


</div>

    </main>
    


<footer class="site-footer">

  <h2 class="footer-heading">seL4 docs</h2>

  <div class="footer-col-wrapper">

    <div class="col-md-2">
      



<ul class="social-media-list">
  <li><a href="https://github.com/sel4"><i class="fab fa-github"></i> <span class="username">sel4</span></a></li><li><a href="https://github.com/sel4proj"><i class="fab fa-github"></i> <span class="username">sel4proj</span></a></li>
</ul>

    </div>

    <div class="col-md-8">
      <ul class="list-unstyled">
        <li>
          This site is for displaying seL4 related documentation.  Pull requests are welcome.
        </li>
        
          <li>
            Site last updated: Fri Feb 7 10:17:38 2025 +1100 ee78c8857c
          </li>
          <li>
                Page last updated: Mon Nov 30 09:25:37 2020 +1100 977ed44f17
          </li>
        
      </ul>
    </div>
    <div class="col-md-2">
<a href="https://github.com/seL4/docs/blob/master/projects/camkes-vm/zmq-samples.md">View page on GitHub</a>
      <br />
      <a href="https://github.com/seL4/docs/edit/master/projects/camkes-vm/zmq-samples.md">Edit page on GitHub</a>
      <br />
      <a href="/sitemap">Sitemap</a>
    </div>

  </div>

</footer>

  </body>
</html>
